# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'Yang'
__mtime__ = '2018/12/19 21:59'
# code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
            ┃      ☃      ┃
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗━━━┓
                ┃  神兽保佑    ┣┓
                ┃　永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
"""
import numpy as np
import pandas as pd
import pymysql
import time
import re
import requests
from bs4 import BeautifulSoup
from sqlalchemy import create_engine

"""
爬去说明：
爬取书籍的简介，作者简介
书籍大纲，和图片的url
顺序：
书籍url，img_url, book_intro, author_intro, book_outline
爬取4个字段，book_outline，author_intro 不一定有 
内容简介必有
"""

# url = 'https://book.douban.com/subject/25862578/'

url = 'https://book.douban.com/subject/30318654/'


# 请求页面，使用BeautifulSoup解析网页代码
def get_soup(url):
    page = requests.get(url.format(0)).text.replace('\n', '')
    page = BeautifulSoup(page, 'lxml')
    return page


db = pymysql.connect(host='localhost', port=3306, user='root', password='123', db='pc')
cur = db.cursor()
with open('url.csv') as f:
    i = 0;
    while True:
        url = f.readline().replace('\n', '')
        if url == '':
            break
        soup = get_soup(url)
        isbn = url.split('/')[-2]
        intro = soup.select('div.intro')
        # intro 一般是3条，如果是3条，第一二条都是书籍简介，第三条是作者简介
        # outline一般是1条，书籍的目录信息 E
        outline = soup.select('#dir_' + isbn + '_full')
        img_url = soup.select('a.nbg')[0].get('href')
        sql_info = []
        sql_info.append(str(url))
        sql_info.append(str(img_url))
        print(sql_info)
        # 1 是内容简介，2是内容简介加作者简介，3可能是作者或书籍其中之一更多选项，4是两个都有一个更多选项
        # 现将3单独用函数判断
        intro = [str(i).replace('\n', '') for i in intro]
        if len(intro) == 2:
            sql_info.append(str(intro[0]))
            sql_info.append(str(intro[1]))
        elif len(intro) == 1:
            sql_info.append(str(intro[0]))
            sql_info.append('')
        elif len(intro) == 3:
            l = [i for i in intro if str(i).find('a_show_full') == -1]
            sql_info.append(str(l[0]))
            sql_info.append(str(l[1]))
        elif len(intro) == 4:
            sql_info.append(str(intro[1]))
            sql_info.append(str(intro[3]))

        outline = [str(i).replace('\n', '') for i in outline]
        if len(outline) == 1:
            sql_info.append(str(outline[0]))
        elif len(outline) == 2:
            sql_info.append(str(outline[1]))
        elif len(outline) == 0:
            sql_info.append('')
        sql_info = [i.replace('\'', '"') for i in sql_info]
        sql = "update douban_book_info set image_url='{0}', book_intro='{1}', book_author_intro='{2}', book_outline='{3}' where book_url='{4}';"
        sql = sql.format(sql_info[1], sql_info[2], sql_info[3], sql_info[4], sql_info[0])
        cursor.execute(sql)
        db.commit()
        time.sleep(1)
        i = i + 1;
        print(i)
db.close()
